# -*- coding: utf-8 -*-

import base64

import scrapy

from ScrapyObject.spiders.utils.url_utils import *
class ComprehensiveVideoSpider(scrapy.Spider):
    name = 'comprehensive_video_spider'
    start_urls = ['https://example.com/videos']
    custom_settings = {
        'DEPTH_LIMIT': 3,  # 限制爬取深度
        'CONCURRENT_REQUESTS': 8,  # 并发请求数
    }

    def parse(self, response):
        # 1. 从HTML标签提取
        selectors = [
            ('video', 'src'),
            ('source', 'src'),
            ('iframe', 'src'),
            ('a', 'href'),  # 可能包含视频链接
            ('embed', 'src')
        ]

        for tag, attr in selectors:
            for element in response.css(tag):
                url = element.attrib.get(attr)
                if url and self.is_video_url(url):
                    yield {
                        'video_url': response.urljoin(url),
                        'tag': tag,
                        'source_url': response.url
                    }

        # 2. 从CSS/JS中提取
        for script in response.css('script::text').getall():
            for url in self.extract_urls_from_text(script):
                if self.is_video_url(url):
                    yield {
                        'video_url': response.urljoin(url),
                        'type': 'script',
                        'source_url': response.url
                    }

        # 3. 跟随其他链接
        if response.meta.get('depth', 0) < self.custom_settings['DEPTH_LIMIT']:
            for link in response.css('a::attr(href)').getall():
                yield response.follow(link, self.parse)

    def is_video_url(self, url):
        video_extensions = ['.mp4', '.webm', '.ogg', '.mov', '.avi', '.flv']
        video_domains = ['youtube.com', 'vimeo.com', 'dailymotion.com']

        return (any(ext in url.lower() for ext in video_extensions) or
                any(domain in url.lower() for domain in video_domains))

